{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "8789b99e",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'transformers'",
     "output_type": "error",
     "traceback": [
      "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
      "\u001b[31mModuleNotFoundError\u001b[39m                       Traceback (most recent call last)",
      "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 3\u001b[39m\n\u001b[32m      1\u001b[39m \u001b[38;5;66;03m# 导入必要的库\u001b[39;00m\n\u001b[32m      2\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mtorch\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m3\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mtransformers\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m AutoModelForCausalLM, AutoTokenizer\n\u001b[32m      4\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mpeft\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m get_peft_model, LoraConfig, TaskType\n\u001b[32m      6\u001b[39m \u001b[38;5;66;03m# 设置设备\u001b[39;00m\n",
      "\u001b[31mModuleNotFoundError\u001b[39m: No module named 'transformers'"
     ]
    }
   ],
   "source": [
    "# 导入必要的库\n",
    "import torch\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "from peft import get_peft_model, LoraConfig, TaskType\n",
    "\n",
    "# 设置设备\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "# 加载基础模型和分词器\n",
    "model_name = \"bert-base-chinese\"  # 可以替换为其他预训练模型\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "base_model = AutoModelForCausalLM.from_pretrained(model_name).to(device)\n",
    "\n",
    "# 配置LoRA参数\n",
    "peft_config = LoraConfig(\n",
    "    task_type=TaskType.CAUSAL_LM,\n",
    "    r=8,  # LoRA秩\n",
    "    lora_alpha=32,  # LoRA缩放\n",
    "    lora_dropout=0.1,  # Dropout概率\n",
    "    bias=\"none\",\n",
    "    target_modules=[\"query\", \"value\"]  # 需要微调的模块\n",
    ")\n",
    "\n",
    "# 创建PEFT模型\n",
    "model = get_peft_model(base_model, peft_config)\n",
    "\n",
    "# 打印可训练参数信息\n",
    "model.print_trainable_parameters()\n",
    "\n",
    "# 准备训练数据\n",
    "def prepare_data():\n",
    "    # 这里添加数据处理逻辑\n",
    "    pass\n",
    "\n",
    "# 训练函数\n",
    "def train(model, tokenizer, train_data, epochs=3):\n",
    "    model.train()\n",
    "    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-4)\n",
    "    \n",
    "    for epoch in range(epochs):\n",
    "        total_loss = 0\n",
    "        for batch in train_data:\n",
    "            # 添加训练循环逻辑\n",
    "            optimizer.zero_grad()\n",
    "            outputs = model(**batch)\n",
    "            loss = outputs.loss\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            total_loss += loss.item()\n",
    "            \n",
    "        print(f\"Epoch {epoch+1}, Average Loss: {total_loss/len(train_data)}\")\n",
    "\n",
    "# 保存模型\n",
    "def save_model(model, output_dir):\n",
    "    model.save_pretrained(output_dir)\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
